{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../FinNLP\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Eastmoney"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from finnlp.data_sources.social_media.eastmoney_streaming import Eastmoney_Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pages = 3\n",
    "stock = \"600519\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading ... 0 1 2 "
     ]
    }
   ],
   "source": [
    "downloader = Eastmoney_Streaming()\n",
    "downloader.download_streaming_stock(stock, pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(241, 92)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "downloader.dataframe.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>post_id</th>\n",
       "      <th>post_title</th>\n",
       "      <th>stockbar_code</th>\n",
       "      <th>stockbar_name</th>\n",
       "      <th>stockbar_type</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_nickname</th>\n",
       "      <th>user_extendinfos</th>\n",
       "      <th>post_click_count</th>\n",
       "      <th>post_forward_count</th>\n",
       "      <th>...</th>\n",
       "      <th>relate_topic</th>\n",
       "      <th>zwpage_flag</th>\n",
       "      <th>source_post_comment_count</th>\n",
       "      <th>post_atuser</th>\n",
       "      <th>reply_list</th>\n",
       "      <th>content_type</th>\n",
       "      <th>repost_state</th>\n",
       "      <th>reptile_state</th>\n",
       "      <th>allow_likes_state</th>\n",
       "      <th>post_is_hot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1324058647</td>\n",
       "      <td>贵州茅台：每股派25.911元 6月30日共计派发现金红利325.49亿元</td>\n",
       "      <td>600519</td>\n",
       "      <td>贵州茅台吧</td>\n",
       "      <td>100.0</td>\n",
       "      <td>7344113638256342</td>\n",
       "      <td>贵州茅台资讯</td>\n",
       "      <td>{'user_accreditinfos': None, 'deactive': '0', ...</td>\n",
       "      <td>3799</td>\n",
       "      <td>14</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 92 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      post_id                             post_title stockbar_code  \\\n",
       "0  1324058647  贵州茅台：每股派25.911元 6月30日共计派发现金红利325.49亿元        600519   \n",
       "\n",
       "  stockbar_name  stockbar_type           user_id user_nickname  \\\n",
       "0         贵州茅台吧          100.0  7344113638256342        贵州茅台资讯   \n",
       "\n",
       "                                    user_extendinfos  post_click_count  \\\n",
       "0  {'user_accreditinfos': None, 'deactive': '0', ...              3799   \n",
       "\n",
       "   post_forward_count  ...  relate_topic zwpage_flag  \\\n",
       "0                  14  ...           NaN         NaN   \n",
       "\n",
       "  source_post_comment_count  post_atuser  reply_list  content_type  \\\n",
       "0                       NaN          NaN         NaN           NaN   \n",
       "\n",
       "  repost_state  reptile_state  allow_likes_state post_is_hot  \n",
       "0          NaN            NaN                NaN         NaN  \n",
       "\n",
       "[1 rows x 92 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "downloader.dataframe.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>post_title</th>\n",
       "      <th>user_nickname</th>\n",
       "      <th>stockbar_name</th>\n",
       "      <th>post_click_count</th>\n",
       "      <th>post_forward_count</th>\n",
       "      <th>post_comment_count</th>\n",
       "      <th>post_publish_time</th>\n",
       "      <th>post_last_time</th>\n",
       "      <th>post_display_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>贵州茅台：每股派25.911元 6月30日共计派发现金红利325.49亿元</td>\n",
       "      <td>贵州茅台资讯</td>\n",
       "      <td>贵州茅台吧</td>\n",
       "      <td>3799</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>2023-06-25 22:17:50</td>\n",
       "      <td>2023-06-26 03:12:47</td>\n",
       "      <td>2023-06-25 22:17:50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>贵州茅台:贵州茅台2022年年度权益分派实施公告</td>\n",
       "      <td>贵州茅台资讯</td>\n",
       "      <td>贵州茅台吧</td>\n",
       "      <td>6423</td>\n",
       "      <td>47</td>\n",
       "      <td>17</td>\n",
       "      <td>2023-06-25 15:32:42</td>\n",
       "      <td>2023-06-26 00:57:39</td>\n",
       "      <td>2023-06-26 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>将派发现金红利325.49亿元！贵州茅台上市以来累计分红超2000亿元</td>\n",
       "      <td>贵州茅台资讯</td>\n",
       "      <td>贵州茅台吧</td>\n",
       "      <td>460</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2023-06-25 23:49:07</td>\n",
       "      <td>2023-06-25 23:49:07</td>\n",
       "      <td>2023-06-25 23:49:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>茅台冰淇淋悄然卖数亿 年轻市场真被抓住了吗</td>\n",
       "      <td>贵州茅台资讯</td>\n",
       "      <td>贵州茅台吧</td>\n",
       "      <td>2612</td>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "      <td>2023-06-24 07:03:53</td>\n",
       "      <td>2023-06-25 18:48:21</td>\n",
       "      <td>2023-06-24 07:03:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>白酒本周跌5.49%原因是什么？下周怎么看？</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10197</td>\n",
       "      <td>4</td>\n",
       "      <td>25</td>\n",
       "      <td>2023-06-24 12:29:53</td>\n",
       "      <td>2023-06-25 23:12:49</td>\n",
       "      <td>2023-06-24 12:29:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>本周持仓与下周交易计划</td>\n",
       "      <td>满仓日记</td>\n",
       "      <td>财富号评论吧</td>\n",
       "      <td>547</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2023-06-25 20:30:54</td>\n",
       "      <td>2023-06-26 03:19:08</td>\n",
       "      <td>2023-06-25 20:30:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>茅台酒的估值真的是高</td>\n",
       "      <td>菩萨小跟班888</td>\n",
       "      <td>贵州茅台吧</td>\n",
       "      <td>33</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2023-06-26 03:02:14</td>\n",
       "      <td>2023-06-26 03:02:14</td>\n",
       "      <td>2023-06-26 03:02:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>茅台里面的资金估计要出来支持一些中小微企业政策导向[吃瓜]</td>\n",
       "      <td>菩萨小跟班888</td>\n",
       "      <td>贵州茅台吧</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2023-06-26 01:50:12</td>\n",
       "      <td>2023-06-26 01:50:12</td>\n",
       "      <td>2023-06-26 01:50:12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>每股市值收益率，还没有银行定期利息高呢。（远离泡沫浮云地震带）</td>\n",
       "      <td>章鱼帝的智慧</td>\n",
       "      <td>贵州茅台吧</td>\n",
       "      <td>33</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2023-06-25 22:48:49</td>\n",
       "      <td>2023-06-26 01:20:04</td>\n",
       "      <td>2023-06-25 22:48:49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>6月最后的倔强（浪潮信息，昆仑万维，鸿博股份）赛道复苏。</td>\n",
       "      <td>夏夏爱美丽</td>\n",
       "      <td>财富号评论吧</td>\n",
       "      <td>2459</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>2023-06-25 22:16:03</td>\n",
       "      <td>2023-06-26 00:45:53</td>\n",
       "      <td>2023-06-25 22:16:03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              post_title user_nickname stockbar_name  \\\n",
       "0  贵州茅台：每股派25.911元 6月30日共计派发现金红利325.49亿元        贵州茅台资讯         贵州茅台吧   \n",
       "1               贵州茅台:贵州茅台2022年年度权益分派实施公告        贵州茅台资讯         贵州茅台吧   \n",
       "2    将派发现金红利325.49亿元！贵州茅台上市以来累计分红超2000亿元        贵州茅台资讯         贵州茅台吧   \n",
       "3                  茅台冰淇淋悄然卖数亿 年轻市场真被抓住了吗        贵州茅台资讯         贵州茅台吧   \n",
       "4                 白酒本周跌5.49%原因是什么？下周怎么看？           NaN           NaN   \n",
       "5                            本周持仓与下周交易计划          满仓日记        财富号评论吧   \n",
       "6                             茅台酒的估值真的是高      菩萨小跟班888         贵州茅台吧   \n",
       "7          茅台里面的资金估计要出来支持一些中小微企业政策导向[吃瓜]      菩萨小跟班888         贵州茅台吧   \n",
       "8        每股市值收益率，还没有银行定期利息高呢。（远离泡沫浮云地震带）        章鱼帝的智慧         贵州茅台吧   \n",
       "9           6月最后的倔强（浪潮信息，昆仑万维，鸿博股份）赛道复苏。         夏夏爱美丽        财富号评论吧   \n",
       "\n",
       "   post_click_count  post_forward_count  post_comment_count  \\\n",
       "0              3799                  14                  15   \n",
       "1              6423                  47                  17   \n",
       "2               460                   1                   0   \n",
       "3              2612                  15                  11   \n",
       "4             10197                   4                  25   \n",
       "5               547                   2                   1   \n",
       "6                33                   0                   0   \n",
       "7                24                   0                   0   \n",
       "8                33                   0                   1   \n",
       "9              2459                   0                  34   \n",
       "\n",
       "     post_publish_time       post_last_time    post_display_time  \n",
       "0  2023-06-25 22:17:50  2023-06-26 03:12:47  2023-06-25 22:17:50  \n",
       "1  2023-06-25 15:32:42  2023-06-26 00:57:39  2023-06-26 00:00:00  \n",
       "2  2023-06-25 23:49:07  2023-06-25 23:49:07  2023-06-25 23:49:07  \n",
       "3  2023-06-24 07:03:53  2023-06-25 18:48:21  2023-06-24 07:03:53  \n",
       "4  2023-06-24 12:29:53  2023-06-25 23:12:49  2023-06-24 12:29:53  \n",
       "5  2023-06-25 20:30:54  2023-06-26 03:19:08  2023-06-25 20:30:54  \n",
       "6  2023-06-26 03:02:14  2023-06-26 03:02:14  2023-06-26 03:02:14  \n",
       "7  2023-06-26 01:50:12  2023-06-26 01:50:12  2023-06-26 01:50:12  \n",
       "8  2023-06-25 22:48:49  2023-06-26 01:20:04  2023-06-25 22:48:49  \n",
       "9  2023-06-25 22:16:03  2023-06-26 00:45:53  2023-06-25 22:16:03  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_columns = [\"post_title\",\"user_nickname\", \"stockbar_name\" ,\"post_click_count\", \"post_forward_count\", \"post_comment_count\", \"post_publish_time\", \"post_last_time\", \"post_display_time\"]\n",
    "downloader.dataframe[selected_columns].head(10)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Facebook get cookies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import json\n",
    "\n",
    "browser = webdriver.ChromiumEdge()\n",
    "browser.get('https://www.facebook.com')\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Please login your account in the brower"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cookies = browser.get_cookies() \n",
    "with open(\"cookies.json\", \"w\", encoding=\"utf-8\") as cks:\n",
    "    json.dump(cookies, cks)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Facebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from finnlp.data_sources.social_media.facebook_streaming import Facebook_Streaming\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load cookies\n",
    "with open(\"cookies.json\", \"r\", encoding=\"utf-8\") as cks: \n",
    "    cookies = json.load(cks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {\n",
    "    \"cookies\":cookies, \n",
    "    \"headless\": False,\n",
    "    \"stealth_path\":\"../../FinNLP/finnlp/data_sources/social_media/stealth.min.js\"\n",
    "    }\n",
    "pages = 3\n",
    "stock = \"AAPL\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 17/17 [00:57<00:00,  3.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Only support the first page now!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "downloader = Facebook_Streaming(config)\n",
    "downloader.download_streaming_stock(stock, pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AAPL (Stock Market)</td>\n",
       "      <td>4h󰞋󰙷</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Day 7\\nIntroduction to Stock Market\\nWhat you ...</td>\n",
       "      <td>6h󰞋󰙷</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>US: AAPL new high and breakout from two-year r...</td>\n",
       "      <td>1d󰞋󰙷</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              content  date\n",
       "6                                 AAPL (Stock Market)  4h󰞋󰙷\n",
       "8   Day 7\\nIntroduction to Stock Market\\nWhat you ...  6h󰞋󰙷\n",
       "11  US: AAPL new high and breakout from two-year r...  1d󰞋󰙷"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "downloader.dataframe"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Xueqiu / 雪球"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from finnlp.data_sources.social_media.xueqiu_streaming import Xueqiu_Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pages = 3\n",
    "stock = \"茅台\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading ... 0 1 2 "
     ]
    }
   ],
   "source": [
    "downloader = Xueqiu_Streaming()\n",
    "downloader.download_streaming_stock(stock, pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(29, 53)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "downloader.dataframe.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>blocked</th>\n",
       "      <th>blocking</th>\n",
       "      <th>canEdit</th>\n",
       "      <th>commentId</th>\n",
       "      <th>controversial</th>\n",
       "      <th>created_at</th>\n",
       "      <th>description</th>\n",
       "      <th>donate_count</th>\n",
       "      <th>donate_snowcoin</th>\n",
       "      <th>editable</th>\n",
       "      <th>...</th>\n",
       "      <th>truncated_by</th>\n",
       "      <th>type</th>\n",
       "      <th>user</th>\n",
       "      <th>user_id</th>\n",
       "      <th>view_count</th>\n",
       "      <th>firstImg</th>\n",
       "      <th>pic_sizes</th>\n",
       "      <th>edited_at</th>\n",
       "      <th>quote_cards</th>\n",
       "      <th>symbol_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2023-06-25 12:15:07</td>\n",
       "      <td>&lt;a href=\"http://xueqiu.com/S/SZ000860\" target=...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>{'allow_all_stock': False, 'block_status': 0, ...</td>\n",
       "      <td>8364804052</td>\n",
       "      <td>471</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 53 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   blocked  blocking  canEdit  commentId  controversial          created_at  \\\n",
       "0    False     False     True          0          False 2023-06-25 12:15:07   \n",
       "\n",
       "                                         description  donate_count  \\\n",
       "0  <a href=\"http://xueqiu.com/S/SZ000860\" target=...             0   \n",
       "\n",
       "   donate_snowcoin  editable  ...  truncated_by  type  \\\n",
       "0                0      True  ...             0     2   \n",
       "\n",
       "                                                user     user_id view_count  \\\n",
       "0  {'allow_all_stock': False, 'block_status': 0, ...  8364804052        471   \n",
       "\n",
       "   firstImg  pic_sizes  edited_at  quote_cards  symbol_id  \n",
       "0       NaN        NaN        NaN          NaN        NaN  \n",
       "\n",
       "[1 rows x 53 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "downloader.dataframe.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>description</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "      <th>source</th>\n",
       "      <th>user</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023-06-25 12:15:07</td>\n",
       "      <td>&lt;a href=\"http://xueqiu.com/S/SZ000860\" target=...</td>\n",
       "      <td></td>\n",
       "      <td>&lt;a href=\"http://xueqiu.com/S/SZ000860\" target=...</td>\n",
       "      <td>/8364804052/253976413</td>\n",
       "      <td>Android</td>\n",
       "      <td>{'allow_all_stock': False, 'block_status': 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2023-06-25 12:14:22</td>\n",
       "      <td>&lt;a href=\"http://xueqiu.com/S/SH600519\" target=...</td>\n",
       "      <td></td>\n",
       "      <td>&lt;p&gt;&lt;a href=\"http://xueqiu.com/S/SH600519\" targ...</td>\n",
       "      <td>/4631817224/253976390</td>\n",
       "      <td>雪球</td>\n",
       "      <td>{'allow_all_stock': False, 'block_status': 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2023-06-25 12:13:01</td>\n",
       "      <td>...提高。白酒：五粮液、迎驾贡酒、&lt;span class='highlight'&gt;茅台&lt;/...</td>\n",
       "      <td>6.25 赛道和白马的机会</td>\n",
       "      <td>&lt;p&gt;这个假期外围的环境不太好，已经是基本共识了。明天开盘大A承压低开也基本是一致预期。这么...</td>\n",
       "      <td>/4322952939/253976335</td>\n",
       "      <td>雪球</td>\n",
       "      <td>{'allow_all_stock': False, 'block_status': 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023-06-25 11:58:55</td>\n",
       "      <td>茅台发生活费了</td>\n",
       "      <td></td>\n",
       "      <td>茅台发生活费了&lt;br/&gt;&lt;img class=\"ke_img\" src=\"https://x...</td>\n",
       "      <td>/4653939718/253975764</td>\n",
       "      <td>iPhone</td>\n",
       "      <td>{'allow_all_stock': False, 'block_status': 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023-06-25 11:54:05</td>\n",
       "      <td>...业绩及股价，形成正反馈。当年&lt;span class='highlight'&gt;茅台&lt;/s...</td>\n",
       "      <td>持仓吹票，共同致富</td>\n",
       "      <td>&lt;p&gt;&lt;a href=\"http://xueqiu.com/k?q=%23%E4%BB%A5...</td>\n",
       "      <td>/8113901491/253975613</td>\n",
       "      <td>Android</td>\n",
       "      <td>{'allow_all_stock': False, 'block_status': 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2023-06-25 11:50:11</td>\n",
       "      <td>微酒酒业快讯，6月25日，酒业新闻一览-·企业动态·-01&lt;span class='high...</td>\n",
       "      <td>6.25：&lt;span class='highlight'&gt;茅&lt;/span&gt;&lt;span cla...</td>\n",
       "      <td>&lt;p&gt;&lt;img class=\"ke_img\" src=\"https://xqimg.imed...</td>\n",
       "      <td>/3615583399/253975485</td>\n",
       "      <td>雪球</td>\n",
       "      <td>{'allow_all_stock': False, 'block_status': 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2023-06-25 11:48:42</td>\n",
       "      <td>&lt;a href=\"http://xueqiu.com/S/SH603027\" target=...</td>\n",
       "      <td></td>\n",
       "      <td>&lt;a href=\"http://xueqiu.com/S/SH603027\" target=...</td>\n",
       "      <td>/2659542807/253975430</td>\n",
       "      <td>iPhone</td>\n",
       "      <td>{'allow_all_stock': False, 'block_status': 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2023-06-25 11:45:54</td>\n",
       "      <td>段永平说：我不鼓励小散投&lt;a href=\"https://xueqiu.com/S/AAPL...</td>\n",
       "      <td></td>\n",
       "      <td>段永平说：我不鼓励小散投&lt;a href=\"https://xueqiu.com/S/AAPL...</td>\n",
       "      <td>/9456980430/253975338</td>\n",
       "      <td>iPhone</td>\n",
       "      <td>{'allow_all_stock': False, 'block_status': 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2023-06-25 11:33:01</td>\n",
       "      <td>泸州老窖酒传统酿制技艺第二十三代传承人·国窖1573·曾娜大师鉴藏版，端午举杯小酒。&lt;br/...</td>\n",
       "      <td></td>\n",
       "      <td>泸州老窖酒传统酿制技艺第二十三代传承人·国窖1573·曾娜大师鉴藏版，端午举杯小酒。&lt;br/...</td>\n",
       "      <td>/9893982765/253974916</td>\n",
       "      <td>Android</td>\n",
       "      <td>{'allow_all_stock': False, 'block_status': 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2023-06-25 11:25:44</td>\n",
       "      <td>...酒店中，白酒卖得最好的往往不是&lt;span class='highlight'&gt;茅台&lt;/...</td>\n",
       "      <td>街头没生意的烟酒店，为什么不会倒闭</td>\n",
       "      <td>&lt;p&gt;&lt;img class=\"ke_img\" src=\"https://xqimg.imed...</td>\n",
       "      <td>/5497522856/253974630</td>\n",
       "      <td>雪球</td>\n",
       "      <td>{'allow_all_stock': False, 'block_status': 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           created_at                                        description  \\\n",
       "0 2023-06-25 12:15:07  <a href=\"http://xueqiu.com/S/SZ000860\" target=...   \n",
       "1 2023-06-25 12:14:22  <a href=\"http://xueqiu.com/S/SH600519\" target=...   \n",
       "2 2023-06-25 12:13:01  ...提高。白酒：五粮液、迎驾贡酒、<span class='highlight'>茅台</...   \n",
       "3 2023-06-25 11:58:55                                            茅台发生活费了   \n",
       "4 2023-06-25 11:54:05  ...业绩及股价，形成正反馈。当年<span class='highlight'>茅台</s...   \n",
       "5 2023-06-25 11:50:11  微酒酒业快讯，6月25日，酒业新闻一览-·企业动态·-01<span class='high...   \n",
       "6 2023-06-25 11:48:42  <a href=\"http://xueqiu.com/S/SH603027\" target=...   \n",
       "7 2023-06-25 11:45:54  段永平说：我不鼓励小散投<a href=\"https://xueqiu.com/S/AAPL...   \n",
       "8 2023-06-25 11:33:01  泸州老窖酒传统酿制技艺第二十三代传承人·国窖1573·曾娜大师鉴藏版，端午举杯小酒。<br/...   \n",
       "9 2023-06-25 11:25:44  ...酒店中，白酒卖得最好的往往不是<span class='highlight'>茅台</...   \n",
       "\n",
       "                                               title  \\\n",
       "0                                                      \n",
       "1                                                      \n",
       "2                                      6.25 赛道和白马的机会   \n",
       "3                                                      \n",
       "4                                          持仓吹票，共同致富   \n",
       "5  6.25：<span class='highlight'>茅</span><span cla...   \n",
       "6                                                      \n",
       "7                                                      \n",
       "8                                                      \n",
       "9                                  街头没生意的烟酒店，为什么不会倒闭   \n",
       "\n",
       "                                                text                 target  \\\n",
       "0  <a href=\"http://xueqiu.com/S/SZ000860\" target=...  /8364804052/253976413   \n",
       "1  <p><a href=\"http://xueqiu.com/S/SH600519\" targ...  /4631817224/253976390   \n",
       "2  <p>这个假期外围的环境不太好，已经是基本共识了。明天开盘大A承压低开也基本是一致预期。这么...  /4322952939/253976335   \n",
       "3  茅台发生活费了<br/><img class=\"ke_img\" src=\"https://x...  /4653939718/253975764   \n",
       "4  <p><a href=\"http://xueqiu.com/k?q=%23%E4%BB%A5...  /8113901491/253975613   \n",
       "5  <p><img class=\"ke_img\" src=\"https://xqimg.imed...  /3615583399/253975485   \n",
       "6  <a href=\"http://xueqiu.com/S/SH603027\" target=...  /2659542807/253975430   \n",
       "7  段永平说：我不鼓励小散投<a href=\"https://xueqiu.com/S/AAPL...  /9456980430/253975338   \n",
       "8  泸州老窖酒传统酿制技艺第二十三代传承人·国窖1573·曾娜大师鉴藏版，端午举杯小酒。<br/...  /9893982765/253974916   \n",
       "9  <p><img class=\"ke_img\" src=\"https://xqimg.imed...  /5497522856/253974630   \n",
       "\n",
       "    source                                               user  \n",
       "0  Android  {'allow_all_stock': False, 'block_status': 0, ...  \n",
       "1       雪球  {'allow_all_stock': False, 'block_status': 0, ...  \n",
       "2       雪球  {'allow_all_stock': False, 'block_status': 0, ...  \n",
       "3   iPhone  {'allow_all_stock': False, 'block_status': 0, ...  \n",
       "4  Android  {'allow_all_stock': False, 'block_status': 0, ...  \n",
       "5       雪球  {'allow_all_stock': False, 'block_status': 0, ...  \n",
       "6   iPhone  {'allow_all_stock': False, 'block_status': 0, ...  \n",
       "7   iPhone  {'allow_all_stock': False, 'block_status': 0, ...  \n",
       "8  Android  {'allow_all_stock': False, 'block_status': 0, ...  \n",
       "9       雪球  {'allow_all_stock': False, 'block_status': 0, ...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_columns = [\"created_at\", \"description\", \"title\", \"text\", \"target\", \"source\", \"user\"]\n",
    "downloader.dataframe[selected_columns].head(10)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stocktwits Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from finnlp.data_sources.social_media.stocktwits_streaming import Stocktwits_Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "pages = 3\n",
    "stock = \"AAPL\"\n",
    "config = {\n",
    "    \"use_proxy\": \"us_free\",\n",
    "    \"max_retry\": 5,\n",
    "    \"proxy_pages\": 2,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checking ips: 100%|██████████| 30/30 [01:07<00:00,  2.24s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get proxy ips: 30.\n",
      "Usable proxy ips: 29.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:05<00:00,  1.68s/it]\n"
     ]
    }
   ],
   "source": [
    "downloader = Stocktwits_Streaming(config)\n",
    "downloader.download_streaming_stock(stock, pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>body</th>\n",
       "      <th>created_at</th>\n",
       "      <th>user</th>\n",
       "      <th>source</th>\n",
       "      <th>symbols</th>\n",
       "      <th>prices</th>\n",
       "      <th>mentioned_users</th>\n",
       "      <th>entities</th>\n",
       "      <th>liked_by_self</th>\n",
       "      <th>reshared_by_self</th>\n",
       "      <th>links</th>\n",
       "      <th>reshare_message</th>\n",
       "      <th>conversation</th>\n",
       "      <th>likes</th>\n",
       "      <th>reshares</th>\n",
       "      <th>network</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>522005335</td>\n",
       "      <td>NANCY PELOSI JUST BOUGHT 10,000 SHARES OF APPL...</td>\n",
       "      <td>2023-04-07T15:24:22Z</td>\n",
       "      <td>{'id': 4744627, 'username': 'JavierAyala', 'na...</td>\n",
       "      <td>{'id': 1149, 'title': 'StockTwits for iOS', 'u...</td>\n",
       "      <td>[{'id': 686, 'symbol': 'AAPL', 'symbol_mic': '...</td>\n",
       "      <td>[{'id': 686, 'symbol': 'AAPL', 'symbol_mic': '...</td>\n",
       "      <td>[]</td>\n",
       "      <td>{'sentiment': None}</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>522004768</td>\n",
       "      <td>$AAPL $SPY  \\n \\nhttps://amp.scmp.com/news/chi...</td>\n",
       "      <td>2023-04-07T15:17:43Z</td>\n",
       "      <td>{'id': 6330207, 'username': 'PlainFacts_2121',...</td>\n",
       "      <td>{'id': 2269, 'title': 'StockTwits Web', 'url':...</td>\n",
       "      <td>[{'id': 686, 'symbol': 'AAPL', 'symbol_mic': '...</td>\n",
       "      <td>[{'id': 686, 'symbol': 'AAPL', 'symbol_mic': '...</td>\n",
       "      <td>[]</td>\n",
       "      <td>{'sentiment': None}</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'title': 'China officials who abused health ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                               body  \\\n",
       "0  522005335  NANCY PELOSI JUST BOUGHT 10,000 SHARES OF APPL...   \n",
       "1  522004768  $AAPL $SPY  \\n \\nhttps://amp.scmp.com/news/chi...   \n",
       "\n",
       "             created_at                                               user  \\\n",
       "0  2023-04-07T15:24:22Z  {'id': 4744627, 'username': 'JavierAyala', 'na...   \n",
       "1  2023-04-07T15:17:43Z  {'id': 6330207, 'username': 'PlainFacts_2121',...   \n",
       "\n",
       "                                              source  \\\n",
       "0  {'id': 1149, 'title': 'StockTwits for iOS', 'u...   \n",
       "1  {'id': 2269, 'title': 'StockTwits Web', 'url':...   \n",
       "\n",
       "                                             symbols  \\\n",
       "0  [{'id': 686, 'symbol': 'AAPL', 'symbol_mic': '...   \n",
       "1  [{'id': 686, 'symbol': 'AAPL', 'symbol_mic': '...   \n",
       "\n",
       "                                              prices mentioned_users  \\\n",
       "0  [{'id': 686, 'symbol': 'AAPL', 'symbol_mic': '...              []   \n",
       "1  [{'id': 686, 'symbol': 'AAPL', 'symbol_mic': '...              []   \n",
       "\n",
       "              entities  liked_by_self  reshared_by_self  \\\n",
       "0  {'sentiment': None}          False             False   \n",
       "1  {'sentiment': None}          False             False   \n",
       "\n",
       "                                               links reshare_message  \\\n",
       "0                                                NaN             NaN   \n",
       "1  [{'title': 'China officials who abused health ...             NaN   \n",
       "\n",
       "  conversation likes reshares network  \n",
       "0          NaN   NaN      NaN     NaN  \n",
       "1          NaN   NaN      NaN     NaN  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = downloader.dataframe\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023-04-07T15:24:22Z</td>\n",
       "      <td>NANCY PELOSI JUST BOUGHT 10,000 SHARES OF APPL...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2023-04-07T15:17:43Z</td>\n",
       "      <td>$AAPL $SPY  \\n \\nhttps://amp.scmp.com/news/chi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2023-04-07T15:17:25Z</td>\n",
       "      <td>$AAPL $GOOG $AMZN  I took a Trump today. \\n\\nH...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023-04-07T15:16:54Z</td>\n",
       "      <td>$SPY $AAPL will take this baby down, time for ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023-04-07T15:11:37Z</td>\n",
       "      <td>$SPY $3T  it ALREADY DID - look at the pre-COV...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2023-04-07T15:10:29Z</td>\n",
       "      <td>$AAPL $QQQ $STUDY We are on to the next one! A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2023-04-07T15:06:00Z</td>\n",
       "      <td>$AAPL was analyzed by 48 analysts. The buy con...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2023-04-07T14:54:29Z</td>\n",
       "      <td>$AAPL both retiring. \\n \\nCraig....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2023-04-07T14:40:06Z</td>\n",
       "      <td>$SPY $QQQ $TSLA $AAPL SPY 500 HAS STARTED🚀😍 BI...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2023-04-07T14:38:57Z</td>\n",
       "      <td>Nancy 🩵 (Tim) $AAPL</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             created_at                                               body\n",
       "0  2023-04-07T15:24:22Z  NANCY PELOSI JUST BOUGHT 10,000 SHARES OF APPL...\n",
       "1  2023-04-07T15:17:43Z  $AAPL $SPY  \\n \\nhttps://amp.scmp.com/news/chi...\n",
       "2  2023-04-07T15:17:25Z  $AAPL $GOOG $AMZN  I took a Trump today. \\n\\nH...\n",
       "3  2023-04-07T15:16:54Z  $SPY $AAPL will take this baby down, time for ...\n",
       "4  2023-04-07T15:11:37Z  $SPY $3T  it ALREADY DID - look at the pre-COV...\n",
       "5  2023-04-07T15:10:29Z  $AAPL $QQQ $STUDY We are on to the next one! A...\n",
       "6  2023-04-07T15:06:00Z  $AAPL was analyzed by 48 analysts. The buy con...\n",
       "7  2023-04-07T14:54:29Z                $AAPL both retiring. \\n \\nCraig....\n",
       "8  2023-04-07T14:40:06Z  $SPY $QQQ $TSLA $AAPL SPY 500 HAS STARTED🚀😍 BI...\n",
       "9  2023-04-07T14:38:57Z                                Nancy 🩵 (Tim) $AAPL"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_columns = [\"created_at\", \"body\"]\n",
    "df[selected_columns].head(10)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reddit Wallstreetbets Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from finnlp.data_sources.social_media.reddit_streaming import Reddit_Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pages = 3\n",
    "config = {\n",
    "    # \"use_proxy\": \"us_free\",\n",
    "    \"max_retry\": 5,\n",
    "    \"proxy_pages\": 2,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading by pages...: 100%|██████████| 3/3 [00:08<00:00,  2.83s/it]\n"
     ]
    }
   ],
   "source": [
    "downloader = Reddit_Streaming(config)\n",
    "downloader.download_streaming_all(pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>numComments</th>\n",
       "      <th>created</th>\n",
       "      <th>score</th>\n",
       "      <th>distinguishType</th>\n",
       "      <th>isLocked</th>\n",
       "      <th>isStickied</th>\n",
       "      <th>thumbnail</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>...</th>\n",
       "      <th>postEventInfo</th>\n",
       "      <th>predictionTournament</th>\n",
       "      <th>reactedFrom</th>\n",
       "      <th>removedBy</th>\n",
       "      <th>removedByCategory</th>\n",
       "      <th>subreddit</th>\n",
       "      <th>suggestedCommentSort</th>\n",
       "      <th>topAwardedType</th>\n",
       "      <th>url</th>\n",
       "      <th>whitelistStatus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>t3_12epaq0</td>\n",
       "      <td>8</td>\n",
       "      <td>1680881974000</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>{'url': 'https://b.thumbs.redditmedia.com/W8hd...</td>\n",
       "      <td>Y’all making me feel like spooderman</td>\n",
       "      <td>ghostwholags</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>t3_zr9v10</td>\n",
       "      <td>0</td>\n",
       "      <td>1671595782000</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>{'url': 'https://b.thumbs.redditmedia.com/dJqb...</td>\n",
       "      <td>Do you track your investments in a spreadsheet...</td>\n",
       "      <td>sharesight</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 100 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id numComments        created score distinguishType isLocked  \\\n",
       "0  t3_12epaq0           8  1680881974000     0            None    False   \n",
       "1   t3_zr9v10           0  1671595782000     2            None     True   \n",
       "\n",
       "  isStickied                                          thumbnail  \\\n",
       "0      False  {'url': 'https://b.thumbs.redditmedia.com/W8hd...   \n",
       "1      False  {'url': 'https://b.thumbs.redditmedia.com/dJqb...   \n",
       "\n",
       "                                               title        author  ...  \\\n",
       "0               Y’all making me feel like spooderman  ghostwholags  ...   \n",
       "1  Do you track your investments in a spreadsheet...    sharesight  ...   \n",
       "\n",
       "  postEventInfo predictionTournament reactedFrom removedBy removedByCategory  \\\n",
       "0           NaN                  NaN         NaN       NaN               NaN   \n",
       "1           NaN                  NaN         NaN       NaN               NaN   \n",
       "\n",
       "  subreddit suggestedCommentSort topAwardedType  url whitelistStatus  \n",
       "0       NaN                  NaN            NaN  NaN             NaN  \n",
       "1       NaN                  NaN            NaN  NaN             NaN  \n",
       "\n",
       "[2 rows x 100 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = downloader.dataframe\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>numComments</th>\n",
       "      <th>created</th>\n",
       "      <th>score</th>\n",
       "      <th>distinguishType</th>\n",
       "      <th>isLocked</th>\n",
       "      <th>isStickied</th>\n",
       "      <th>thumbnail</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>...</th>\n",
       "      <th>postEventInfo</th>\n",
       "      <th>predictionTournament</th>\n",
       "      <th>reactedFrom</th>\n",
       "      <th>removedBy</th>\n",
       "      <th>removedByCategory</th>\n",
       "      <th>subreddit</th>\n",
       "      <th>suggestedCommentSort</th>\n",
       "      <th>topAwardedType</th>\n",
       "      <th>url</th>\n",
       "      <th>whitelistStatus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>t3_12epaq0</td>\n",
       "      <td>8</td>\n",
       "      <td>2023-04-07 15:39:34</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>{'url': 'https://b.thumbs.redditmedia.com/W8hd...</td>\n",
       "      <td>Y’all making me feel like spooderman</td>\n",
       "      <td>ghostwholags</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>t3_zr9v10</td>\n",
       "      <td>0</td>\n",
       "      <td>2022-12-21 04:09:42</td>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>{'url': 'https://b.thumbs.redditmedia.com/dJqb...</td>\n",
       "      <td>Do you track your investments in a spreadsheet...</td>\n",
       "      <td>sharesight</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 100 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id numComments             created score distinguishType isLocked  \\\n",
       "0  t3_12epaq0           8 2023-04-07 15:39:34     0            None    False   \n",
       "1   t3_zr9v10           0 2022-12-21 04:09:42     2            None     True   \n",
       "\n",
       "  isStickied                                          thumbnail  \\\n",
       "0      False  {'url': 'https://b.thumbs.redditmedia.com/W8hd...   \n",
       "1      False  {'url': 'https://b.thumbs.redditmedia.com/dJqb...   \n",
       "\n",
       "                                               title        author  ...  \\\n",
       "0               Y’all making me feel like spooderman  ghostwholags  ...   \n",
       "1  Do you track your investments in a spreadsheet...    sharesight  ...   \n",
       "\n",
       "  postEventInfo predictionTournament reactedFrom removedBy removedByCategory  \\\n",
       "0           NaN                  NaN         NaN       NaN               NaN   \n",
       "1           NaN                  NaN         NaN       NaN               NaN   \n",
       "\n",
       "  subreddit suggestedCommentSort topAwardedType  url whitelistStatus  \n",
       "0       NaN                  NaN            NaN  NaN             NaN  \n",
       "1       NaN                  NaN            NaN  NaN             NaN  \n",
       "\n",
       "[2 rows x 100 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df[\"created\"] = pd.to_datetime(df[\"created\"], unit = \"ms\")\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2023-04-07 15:39:34</td>\n",
       "      <td>Y’all making me feel like spooderman</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2022-12-21 04:09:42</td>\n",
       "      <td>Do you track your investments in a spreadsheet...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2022-12-21 04:09:42</td>\n",
       "      <td>Do you track your investments in a spreadsheet...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2023-04-07 15:29:23</td>\n",
       "      <td>Can a Blackberry holder get some help 🥺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2023-04-07 14:49:55</td>\n",
       "      <td>The week of CPI and FOMC Minutes… 4-6-23 SPY/ ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2023-04-07 14:19:22</td>\n",
       "      <td>Well let’s hope your job likes you, thanks Jerome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2023-04-07 14:06:32</td>\n",
       "      <td>Does anyone else feel an overwhelming sense of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2023-04-07 13:47:59</td>\n",
       "      <td>Watermarked Jesus explains the market being cl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2023-04-07 13:26:23</td>\n",
       "      <td>Jobs report shows 236,000 gain in March. Hot l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2023-04-07 13:07:15</td>\n",
       "      <td>The recession is over! Let's buy more stocks!</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              created                                              title\n",
       "0 2023-04-07 15:39:34               Y’all making me feel like spooderman\n",
       "1 2022-12-21 04:09:42  Do you track your investments in a spreadsheet...\n",
       "2 2022-12-21 04:09:42  Do you track your investments in a spreadsheet...\n",
       "3 2023-04-07 15:29:23            Can a Blackberry holder get some help 🥺\n",
       "4 2023-04-07 14:49:55  The week of CPI and FOMC Minutes… 4-6-23 SPY/ ...\n",
       "5 2023-04-07 14:19:22  Well let’s hope your job likes you, thanks Jerome\n",
       "6 2023-04-07 14:06:32  Does anyone else feel an overwhelming sense of...\n",
       "7 2023-04-07 13:47:59  Watermarked Jesus explains the market being cl...\n",
       "8 2023-04-07 13:26:23  Jobs report shows 236,000 gain in March. Hot l...\n",
       "9 2023-04-07 13:07:15      The recession is over! Let's buy more stocks!"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_columns = [\"created\", \"title\"]\n",
    "df[selected_columns].head(10)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weibo Date Range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from finnlp.data_sources.social_media.weibo_date_range import Weibo_Date_Range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_date = \"2016-01-01\"\n",
    "end_date = \"2016-01-02\"\n",
    "stock = \"茅台\"\n",
    "config = {\n",
    "    \"use_proxy\": \"china_free\",\n",
    "    \"max_retry\": 5,\n",
    "    \"proxy_pages\": 5,\n",
    "    \"cookies\": \"Your_Login_Cookies\",\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Gathering free ips by pages...: 100%|██████████| 5/5 [00:09<00:00,  1.95s/it]\n",
      "Checking ips: 100%|██████████| 75/75 [01:23<00:00,  1.11s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "获取到的代理ip数量: 75 。Get proxy ips: 75.\n",
      "能用的代理数量： 13。Usable proxy ips: 13.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading by dates...: 100%|██████████| 2/2 [01:03<00:00, 31.56s/it]\n"
     ]
    }
   ],
   "source": [
    "downloader = Weibo_Date_Range(config)\n",
    "downloader.download_date_range_stock(start_date, end_date, stock = stock)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>date_content</th>\n",
       "      <th>source</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>2016年01月01日23:41</td>\n",
       "      <td>Moto X</td>\n",
       "      <td>#舆论之锤#唯品会发声明证实销售假茅台-手机腾讯网O网页链接分享来自浏览器！</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>2016年01月01日22:57</td>\n",
       "      <td>新浪博客</td>\n",
       "      <td>2016元旦节快乐酒粮网官方新品首发，茅台镇老酒，酱香原浆酒：酒粮网茅台镇白酒酱香老酒纯粮原...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>2016年01月01日22:56</td>\n",
       "      <td>新浪博客</td>\n",
       "      <td>2016元旦节快乐酒粮网官方新品首发，茅台镇老酒，酱香原浆酒：酒粮网茅台镇白酒酱香老酒纯粮原...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>2016年01月01日22:40</td>\n",
       "      <td>五蕴皆崆Android</td>\n",
       "      <td>开心，今天喝了两斤酒（茅台+扎二）三个人，开心！</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>一家专卖假货的网站某宝，你该学学了！//【唯品会售假茅台：供货商被刑拘顾客获十倍补偿】O唯品...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>一家专卖假货的网站//【唯品会售假茅台：供货商被刑拘顾客获十倍补偿】O唯品会售假茅台：供货商...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>2016年01月01日21:46</td>\n",
       "      <td>360安全浏览器</td>\n",
       "      <td>前几天说了几点不看好茅台的理由，今年过节喝点茅台支持下，个人口感，茅台比小五好喝，茅台依然是...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>2016年01月01日21:44</td>\n",
       "      <td>华为P8</td>\n",
       "      <td>老杜酱酒已到货，从明天起正式在甘肃武威开卖。可以不相信我说的话，但一定不要怀疑@杜子建的为人...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>2016年01月01日21:24</td>\n",
       "      <td>华为Ascend P7</td>\n",
       "      <td>【唯品会售假茅台后续：供货商被刑拘顾客获十倍补偿】此前，有网友投诉其在唯品会购买的茅台酒质量...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>2016年01月01日21:16</td>\n",
       "      <td>实得惠省钱网</td>\n",
       "      <td>唯品会卖假茅台，供货商被刑拘，买家获十倍补偿8888元|此前，有网友在网络论坛发贴（唯品会宣...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          date      date_content       source  \\\n",
       "0   2016-01-01  2016年01月01日23:41       Moto X   \n",
       "2   2016-01-01  2016年01月01日22:57         新浪博客   \n",
       "6   2016-01-01  2016年01月01日22:56         新浪博客   \n",
       "17  2016-01-01  2016年01月01日22:40  五蕴皆崆Android   \n",
       "18  2016-01-01               NaN          NaN   \n",
       "19  2016-01-01               NaN          NaN   \n",
       "20  2016-01-01  2016年01月01日21:46     360安全浏览器   \n",
       "21  2016-01-01  2016年01月01日21:44         华为P8   \n",
       "22  2016-01-01  2016年01月01日21:24  华为Ascend P7   \n",
       "23  2016-01-01  2016年01月01日21:16       实得惠省钱网   \n",
       "\n",
       "                                              content  \n",
       "0              #舆论之锤#唯品会发声明证实销售假茅台-手机腾讯网O网页链接分享来自浏览器！  \n",
       "2   2016元旦节快乐酒粮网官方新品首发，茅台镇老酒，酱香原浆酒：酒粮网茅台镇白酒酱香老酒纯粮原...  \n",
       "6   2016元旦节快乐酒粮网官方新品首发，茅台镇老酒，酱香原浆酒：酒粮网茅台镇白酒酱香老酒纯粮原...  \n",
       "17                           开心，今天喝了两斤酒（茅台+扎二）三个人，开心！  \n",
       "18  一家专卖假货的网站某宝，你该学学了！//【唯品会售假茅台：供货商被刑拘顾客获十倍补偿】O唯品...  \n",
       "19  一家专卖假货的网站//【唯品会售假茅台：供货商被刑拘顾客获十倍补偿】O唯品会售假茅台：供货商...  \n",
       "20  前几天说了几点不看好茅台的理由，今年过节喝点茅台支持下，个人口感，茅台比小五好喝，茅台依然是...  \n",
       "21  老杜酱酒已到货，从明天起正式在甘肃武威开卖。可以不相信我说的话，但一定不要怀疑@杜子建的为人...  \n",
       "22  【唯品会售假茅台后续：供货商被刑拘顾客获十倍补偿】此前，有网友投诉其在唯品会购买的茅台酒质量...  \n",
       "23  唯品会卖假茅台，供货商被刑拘，买家获十倍补偿8888元|此前，有网友在网络论坛发贴（唯品会宣...  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = downloader.dataframe\n",
    "df = df.drop_duplicates()\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60, 4)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weibo Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from finnlp.data_sources.social_media.weibo_streaming import Weibo_Streaming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "rounds = 3\n",
    "stock = \"茅台\"\n",
    "config = {\n",
    "    \"use_proxy\": \"china_free\",\n",
    "    \"max_retry\": 5,\n",
    "    \"proxy_pages\": 5,\n",
    "    \"cookies\": \"Your_Login_Cookies\",\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Gathering free ips by pages...: 100%|██████████| 5/5 [00:09<00:00,  1.98s/it]\n",
      "Checking ips: 100%|██████████| 75/75 [01:26<00:00,  1.15s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "获取到的代理ip数量: 75 。Get proxy ips: 75.\n",
      "能用的代理数量： 19。Usable proxy ips: 19.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing the text content and downloading the full passage...: 100%|██████████| 9/9 [00:00<00:00, 64.89it/s]\n",
      "Processing the text content and downloading the full passage...: 100%|██████████| 10/10 [00:09<00:00,  1.07it/s]\n",
      "Processing the text content and downloading the full passage...: 100%|██████████| 10/10 [00:02<00:00,  4.93it/s]\n",
      "Downloading by page..: 100%|██████████| 3/3 [00:19<00:00,  6.46s/it]\n"
     ]
    }
   ],
   "source": [
    "downloader = Weibo_Streaming(config)\n",
    "downloader.download_streaming_stock(stock = stock, rounds = rounds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>card_type</th>\n",
       "      <th>display_followbtn</th>\n",
       "      <th>mblog</th>\n",
       "      <th>itemid</th>\n",
       "      <th>actionlog</th>\n",
       "      <th>cate_id</th>\n",
       "      <th>display_arrow</th>\n",
       "      <th>show_type</th>\n",
       "      <th>scheme</th>\n",
       "      <th>container_color</th>\n",
       "      <th>container_color_dark</th>\n",
       "      <th>content_short</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>{'attitudes_count': 0, 'can_edit': False, 'com...</td>\n",
       "      <td>seqid:187118896|type:61|t:|pos:1-0-0|q:茅台|srid...</td>\n",
       "      <td>{'act_code': 554, 'ext': 'seqid:187118896|type...</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>https://m.weibo.cn/status/MAWMprpPp?mblogid=MA...</td>\n",
       "      <td>#EEEEEE</td>\n",
       "      <td>#151515</td>\n",
       "      <td>事情做好做精，还可以赚大钱的生意才是好生意，而不是忙忙碌碌，最后一算账没赚多少！比如苹果的市...</td>\n",
       "      <td>事情做好做精，还可以赚大钱的生意才是好生意，而不是忙忙碌碌，最后一算账没赚多少！比如苹果的市...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>{'attitudes_count': 0, 'can_edit': False, 'com...</td>\n",
       "      <td>seqid:187118896|type:61|t:|pos:1-0-1|q:茅台|srid...</td>\n",
       "      <td>{'act_code': 554, 'ext': 'seqid:187118896|type...</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>https://m.weibo.cn/status/MAWHVDm0H?mblogid=MA...</td>\n",
       "      <td>#EEEEEE</td>\n",
       "      <td>#151515</td>\n",
       "      <td>茅台茅台成都收4瓶飞天，自提</td>\n",
       "      <td>茅台茅台成都收4瓶飞天，自提</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   card_type  display_followbtn  \\\n",
       "0          9              False   \n",
       "1          9              False   \n",
       "\n",
       "                                               mblog  \\\n",
       "0  {'attitudes_count': 0, 'can_edit': False, 'com...   \n",
       "1  {'attitudes_count': 0, 'can_edit': False, 'com...   \n",
       "\n",
       "                                              itemid  \\\n",
       "0  seqid:187118896|type:61|t:|pos:1-0-0|q:茅台|srid...   \n",
       "1  seqid:187118896|type:61|t:|pos:1-0-1|q:茅台|srid...   \n",
       "\n",
       "                                           actionlog cate_id  display_arrow  \\\n",
       "0  {'act_code': 554, 'ext': 'seqid:187118896|type...      31              0   \n",
       "1  {'act_code': 554, 'ext': 'seqid:187118896|type...      31              0   \n",
       "\n",
       "   show_type                                             scheme  \\\n",
       "0          1  https://m.weibo.cn/status/MAWMprpPp?mblogid=MA...   \n",
       "1          1  https://m.weibo.cn/status/MAWHVDm0H?mblogid=MA...   \n",
       "\n",
       "  container_color container_color_dark  \\\n",
       "0         #EEEEEE              #151515   \n",
       "1         #EEEEEE              #151515   \n",
       "\n",
       "                                       content_short  \\\n",
       "0  事情做好做精，还可以赚大钱的生意才是好生意，而不是忙忙碌碌，最后一算账没赚多少！比如苹果的市...   \n",
       "1                                    茅台茅台成都收4瓶飞天，自提    \n",
       "\n",
       "                                             content  \n",
       "0  事情做好做精，还可以赚大钱的生意才是好生意，而不是忙忙碌碌，最后一算账没赚多少！比如苹果的市...  \n",
       "1                                    茅台茅台成都收4瓶飞天，自提   "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = downloader.dataframe\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>content_short</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>事情做好做精，还可以赚大钱的生意才是好生意，而不是忙忙碌碌，最后一算账没赚多少！比如苹果的市...</td>\n",
       "      <td>事情做好做精，还可以赚大钱的生意才是好生意，而不是忙忙碌碌，最后一算账没赚多少！比如苹果的市...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>茅台茅台成都收4瓶飞天，自提</td>\n",
       "      <td>茅台茅台成都收4瓶飞天，自提</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>我可太喜欢茅台这个防伪了</td>\n",
       "      <td>我可太喜欢茅台这个防伪了</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>没想到 4S店的二楼 是卖茅台的吧</td>\n",
       "      <td>没想到 4S店的二楼 是卖茅台的吧</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>买不起茅台，砸锅卖铁也得买得起茅台冰淇淋 许昌·胖东来时代广场</td>\n",
       "      <td>买不起茅台，砸锅卖铁也得买得起茅台冰淇淋 许昌·胖东来时代广场</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>xxx给我枇杷xxx给我蜂蜜 xxx偷茅台喝（假的）。我很喜欢自己家的产品，感觉很无害纯天然...</td>\n",
       "      <td>xxx给我枇杷xxx给我蜂蜜 xxx偷茅台喝（假的）。我很喜欢自己家的产品，感觉很无害纯天然...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>茅台 奎屯出一只兔茅</td>\n",
       "      <td>茅台 奎屯出一只兔茅</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2022胡润酒类品牌榜发布 2022胡润酒类品牌榜发布点评：与我印象中的有点出入。不出茅台和...</td>\n",
       "      <td>2022胡润酒类品牌榜发布 2022胡润酒类品牌榜发布点评：与我印象中的有点出入。不出茅台和...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>41岁，很美妙！“爸爸生日快乐，吃个蛋糕🍰”小奶音听着上头。爱人，亲戚，朋友，草莓🍓，茅台+...</td>\n",
       "      <td>41岁，很美妙！“爸爸生日快乐，吃个蛋糕🍰”小奶音听着上头。爱人，亲戚，朋友，草莓🍓，茅台+...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>吃到了茅台冰激淋也</td>\n",
       "      <td>吃到了茅台冰激淋也</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       content_short  \\\n",
       "0  事情做好做精，还可以赚大钱的生意才是好生意，而不是忙忙碌碌，最后一算账没赚多少！比如苹果的市...   \n",
       "1                                    茅台茅台成都收4瓶飞天，自提    \n",
       "2                                      我可太喜欢茅台这个防伪了    \n",
       "3                                 没想到 4S店的二楼 是卖茅台的吧    \n",
       "4                   买不起茅台，砸锅卖铁也得买得起茅台冰淇淋 许昌·胖东来时代广场    \n",
       "5  xxx给我枇杷xxx给我蜂蜜 xxx偷茅台喝（假的）。我很喜欢自己家的产品，感觉很无害纯天然...   \n",
       "6                                        茅台 奎屯出一只兔茅    \n",
       "7  2022胡润酒类品牌榜发布 2022胡润酒类品牌榜发布点评：与我印象中的有点出入。不出茅台和...   \n",
       "8  41岁，很美妙！“爸爸生日快乐，吃个蛋糕🍰”小奶音听着上头。爱人，亲戚，朋友，草莓🍓，茅台+...   \n",
       "0                                         吃到了茅台冰激淋也    \n",
       "\n",
       "                                             content  \n",
       "0  事情做好做精，还可以赚大钱的生意才是好生意，而不是忙忙碌碌，最后一算账没赚多少！比如苹果的市...  \n",
       "1                                    茅台茅台成都收4瓶飞天，自提   \n",
       "2                                      我可太喜欢茅台这个防伪了   \n",
       "3                                 没想到 4S店的二楼 是卖茅台的吧   \n",
       "4                   买不起茅台，砸锅卖铁也得买得起茅台冰淇淋 许昌·胖东来时代广场   \n",
       "5  xxx给我枇杷xxx给我蜂蜜 xxx偷茅台喝（假的）。我很喜欢自己家的产品，感觉很无害纯天然...  \n",
       "6                                        茅台 奎屯出一只兔茅   \n",
       "7  2022胡润酒类品牌榜发布 2022胡润酒类品牌榜发布点评：与我印象中的有点出入。不出茅台和...  \n",
       "8  41岁，很美妙！“爸爸生日快乐，吃个蛋糕🍰”小奶音听着上头。爱人，亲戚，朋友，草莓🍓，茅台+...  \n",
       "0                                         吃到了茅台冰激淋也   "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_columns = [\"content_short\", \"content\"]\n",
    "df[selected_columns].head(10)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "finrl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
